import requests
from lxml import etree
import json


class BtcSpider:

    def __init__(self):
        self.url = "https://www.chainnode.com/forum/1"
        self.header = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"
        }
        self.data_list = []

    # 1.发请求-获取源码
    def get_response(self, url):
        response = requests.get(url, headers=self.header)
        data = response.content.decode('utf-8')
        return data

    # 2.解析数据
    def parse_data(self, data):
        # 使用xpath解析当前页面所有的新闻title和url并保存
        # 1.转类型
        x_data = etree.HTML(data)
        # 2.根据xpath路径解析 #模糊查询 //div[contains(@id,"normalthread")]
        result_list_text = x_data.xpath('//a[@class="link-dark-major font-bold bbt-block"]/text()')
        result_list_url = x_data.xpath('//a[@class="link-dark-major font-bold bbt-block"]/@href')
        new_list_text = []
        for i in result_list_text:
            new_list_text.append(i.replace('\n', '').replace(' ' * 22, ''))
        for index, title in enumerate(new_list_text):
            news = {}
            news['title'] = title
            news['url'] = "https://www.chainnode.com" + result_list_url[index]
            self.data_list.append(news)

    # 3.保存数据
    def save_data(self):
        # 将list-str
        data_str = json.dumps(self.data_list)
        with open('btc.json', mode='w', encoding='utf-8') as file:
            file.write(data_str)

    # 4.启动
    def run(self):
        for i in range(1, 5):
            # 1.拼接完整的url
            link_url = self.url + '-%s' % str(i)
            # 2.发请求
            data = self.get_response(link_url)

            # 3.做解析
            self.parse_data(data)

        # 4.保存
        self.save_data()

        with open('btc.json', mode='r', encoding='utf-8') as f:
            ret = f.read()
            print(json.loads(ret))


BtcSpider().run()
